package com.xy.annotaion.handler.kotlin

import us.codecraft.webmagic.Page
import us.codecraft.webmagic.Site
import us.codecraft.webmagic.Spider
import us.codecraft.webmagic.processor.PageProcessor
import us.codecraft.webmagic.processor.example.GithubRepoPageProcessor

class GithubRepoPageProcessor : PageProcessor {
    override fun process(p0: Page?) {
        val page = p0
        p(page)
        page?.addTargetRequests(page?.getHtml()?.links()?.regex("(https://github\\.com/[\\w\\-]+/[\\w\\-]+)")?.all())
        page?.addTargetRequests(page.getHtml().links().regex("(https://github\\.com/[\\w\\-])").all())
        page?.putField("author", page.getUrl().regex("https://github\\.com/(\\w+)/.*").toString())
        page?.putField("name", page.getHtml().xpath("//h1[@class='public']/strong/a/text()").toString())
        if (page?.getResultItems()?.get<Any>("name") == null) {
            page?.setSkip(true)
        }

        page?.putField("readme", page.getHtml().xpath("//div[@id='readme']/tidyText()"))
        p(13)
    }

    override fun getSite(): Site {
        return Site.me().setRetryTimes(3).setSleepTime(1000)
    }

}

fun main() {
    Spider
        .create(GithubRepoPageProcessor())
        .addUrl("https://www.baidu.com")
        .thread(5)
        .run()

}